from lxml import etree
import requests
import csv
from my_fake_useragent import UserAgent
from bs4 import BeautifulSoup
import pymongo

client  = pymongo.MongoClient('localhost',27017)
mydb = client['douban']
test = mydb['top']
ua = UserAgent()
headers = {'User-Agent':ua.random}
print(headers)
fp = open('./top250.csv','wt',newline='',encoding='utf-8')
writer = csv.writer(fp)
writer.writerow(
    ('name','url','author','publisher','date','price','rate','comment')
)
urls = ['https://book.douban.com/top250?start={}'.format(number)
        for number in range(1,250,25)]

for url in urls:
    html = requests.get(url,headers=headers)
    selector = etree.HTML(html.text)
    infos = selector.xpath('//tr[@class="item"]')
    for info in infos:
        name = info.xpath('td/div/a/@title')[0]
        book_url = info.xpath('td/div/a/@href')[0]
        book_infos = info.xpath('td/p/text()')[0]
        author = book_infos.split('/')[0]
        publisher = book_infos.split('/')[-3]
        date = book_infos.split('/')[-2]
        price = book_infos.split('/')[-1]
        rate = info.xpath('td/div/span[2]/text()')[0]
        comments = info.xpath('td/p[2]/span/text()')
        comment = comments[0] if len(comments) != 0 else '空'
        data = {
            'name': name,
            'bok_url': book_url,
            'author': author,
            'publisher': publisher,
            'date': date,
            'price': price,
            'rate': rate,
            'comment': comment
        }
        test.insert_one(data)
        writer.writerow((name,book_url,author,publisher,date,price,rate,comment))
fp.close()


